#!/usr/bin/python
# -*- coding: utf-8 -*-

import feedparser
 
class Collector:
	
	#def __init__(self):
	 
	def parser(self, xmlsrclst, update=0):
		
		entrylist = []
		resultlist = []
		
		if update == 0:
			xmlsrc = xmlsrclst[-2]
			d = feedparser.parse(xmlsrc)
			entry = d.entries
		else:
			entry=xmlsrclst
		for i in xrange(len(entry)):
			
			if entry[i].has_key("content"): 
				content = entry[i].content
				entrylist.append(content[0].value)
			elif entry[i].has_key("summary_detail"): 
				content = entry[i].summary_detail
				entrylist.append(content.value)
			elif entry[i].has_key("summary"): 
				content = entry[i].summary
				entrylist.append(content.value)
			else: content = ""
			try:
				updated = entry[i].updated
			except:
				updated = None
			
			entrylist.append(updated)
			
			try:
				title = entry[i].title
			except:
				title = None
			
			entrylist.append(title)
			
			author = xmlsrclst[-1]
			entrylist.append(author)
				
			try:
				link = entry[i].link
			except:
				link = None
				
			entrylist.append(link)
				
			resultlist.append(entrylist)
			entrylist = []
			categorylist = []	
			
		return resultlist 
		
	def createfiles(self, resultlist, check=0):
		
		try:
			import os
			os.mkdir("files")
			os.chdir("files")
		except:
			os.chdir("files")
		
		for lst in resultlist:
				
			filename = lst[-1]
			print filename
				
			filename = filename.replace("/","--")
					
			if check != 0:
				filename = filename + "girdi"
					
			f = open(filename, "w")
				
			if lst[-2] == None: lst[-2] = "İsimsiz Yazar"
			f.write(lst[-2])
			f.write("\n")
			if lst[-3] == None: lst[-3] = "Belirtilmemiş Başlık"
			f.write(lst[-3])
			f.write("\n")
			f.write(lst[-4])
			f.write("\n")
				
			if check != 0:
				f.write(lst[0])
			else:
				f.write(self.antisanitize(lst[0]))
			f.write("\n")
			f.close()
		os.chdir("..")
		
	
	def openxmlsrc(self, filename):
		rslt=[]
		f = open(filename, "r")
		lst = f.readlines()
		li = [x.split(" ") for x in lst]
		for x in li:
			l = [i.strip() for i in x]
			rslt.append(l)
		return rslt
		
	
	def antisanitize(self, strng):
		sanitizelst = ["a", "abbr", "acronym", "address", "area", "b", "big", "blockquote", "br", "button", "caption", "center", "cite", "code", "col", "colgroup", "dd", "del", "dfn", "dir", "div", "dl", "dt", "em", "fieldset", "font", "form", "h1", "h2", "h3", "h4", "h5", "h6", "hr", "i", "img", "input", "ins", "kbd", "label", "legend", "li", "map", "menu", "ol", "optgroup", "option", "p", "pre", "q", "s", "samp", "select", "small", "span", "strike", "strong", "sub", "sup", "table", "tbody", "td", "textarea", "tfoot", "th", "thead", "tr", "tt", "u", "ul", "var"]
		
		for x in sanitizelst:	
			
			y = "<"+x
			
			while strng.find(y) != -1:
				start = strng.find(y)
				end = strng.find(">", start)
				s = strng[start:end+1]
				
				strng = strng.replace(s, " ")
		
		closetaglist = ["</"+x+">" for x in sanitizelst]
		
		for x in closetaglist:
			strng = strng.replace(x, " ")
		
		strng = strng.replace("<br />", " ")
		strng = strng.replace("<br/>", " ")

		return strng
				
	
	def checker(self, xmlsrclst):
		
		print xmlsrclst
		
		d = feedparser.parse(xmlsrclst[2])
		entry1 = d.entries
		
		date1 = entry1[0].updated
		link1 = entry1[0].link
		
		e = feedparser.parse(xmlsrclst[0])
		
		entry2 = e.entries
		date2 = entry2[0].updated
		link2 = entry2[0].link
		
		if date1 == date2 and link1 == link2: ##i am not interested with the reedited entires
			return False 
		else:
			return True
	
	def update(self, xmlsrclst):## take the new entries only excet from the ones in xml source
		
		entry=[]
		
		d=feedparser.parse(xmlsrclst[0])
		exentry=d.entries
		exdate=exentry[0].updated
		
		dd=feedparser.parse(xmlsrclst[2])
		newentry=dd.entries
		
		for x in newentry:
			if x.updated == exdate: break
			else: entry.append(x)
		
		import os 
		
		lst=xmlsrclst[0].split(xmlsrclst[1])
		filename=lst[1].split("/")
		
		os.chdir(lst[0]+xmlsrclst[1])
		import datetime
		time=datetime.datetime.now()
		suffix="-"+str(time.month)+":"+str(time.day)+"-"+str(time.hour)+":"+str(time.minute)
		os.popen("mv "+filename[1]+" "+filename[1]+suffix)
		os.popen("wget "+xmlsrclst[2]) ## i always download the currecnt xml source but keep the old entires also
		
		os.chdir("..")
		os.chdir("..")
		
		return entry
#show me the number of files under each category so i will see whether there are enough files for training	
	class Counter:
		def __init__(self):
			self.dct={}
		
		def count(self,dirs):
			##print dirs
			import os
			dirfiler = os.walk(os.path.join(os.getcwd(),dirs))
			##print dirfiler
			for x in dirfiler:
				#print x
				if x[-1] == []: continue
				else:
					li = x[0].split(dirs+"/")
					self.dct[li[-1]]=len(x[-1])
			return self.dct
		
	#def cfgread(self,file):
		
		#f=open(file,"r")
		#cfg=f.readlines()
		#rslt=[]
		#for x in cfg:
			#li=x.strip().split(":")
			#if len(li)>1:
				#rslt.append(li[1].strip())		
		#return rslt
		
	def sort_by_value(self,d):
    		""" Returns the keys of dictionary d sorted by their values """
    		items=d.items()
   		backitems=[ [v[1],v[0]] for v in items]
    		backitems.sort()
    		return [backitems[i][1] for i in range(0,len(backitems))]
			
if __name__=="__main__":
	
	import collector
	c = collector.Collector()
	lst = c.openxmlsrc("xml-resources")
	#freq = c.cfgread("venga.cfg")
	
	#import time
	
	#while(True):
		
		#flag = True
	
		#if freq[0]=="day":
			#sleeptime=int(2)##int(freq[1])*24*60*60##freq[1]*24*60*60 tru thing
	for files in lst:
			
		if files == ['']: continue ## in the xml-resources new line at the end of file couses an empty list
			
			#if flag == True:
		src = c.parser(files)
		c.createfiles(src)
	
	print ("Dizinler ve içlerindeki dosya sayıları")
	
	count = c.Counter()
	lst = count.count("files")
	import os
	os.getcwd()
	li = c.sort_by_value(lst)
	for i in xrange(0,len(li),3):
		if i < len(li)-2:
			key = li[i]
			print str(key).ljust(10), "=", str(lst[key]).rjust(5), "".center(10), str(li[i+1]).ljust(10), "=", str(lst[li[i+1]]).rjust(5), "".center(10), str(li[i+2]).ljust(10), "=", str(lst[li[i+2]]).rjust(5)
		
		
			##flag == False
			
			#if(c.checker(files)):
				#print "girdi" 
				#newentrylst=c.update(files)
				#src = c.parser(newentrylst,1)
				#c.createfiles(src)
		#ch=raw_input("Kesmek ister misin (y/n ?)")
		
		#if ch == "y":
			#import sys
			#sys.exit(1)
		#time.sleep(float(sleeptime))
	

